package readData;

import edu.stanford.nlp.process.DocumentPreprocessor;
import java.io.Reader;
import java.io.StringReader;
import java.util.List;

public class Tokenizer {

    public String tokenize(String text) {
        String tokenizedText = "";
        Reader reader = new StringReader(text);
        DocumentPreprocessor dp = new DocumentPreprocessor(reader);
        for (List s : dp) {
            for (Object object : s) {
                tokenizedText += object.toString() + " ";
            }
        }
        tokenizedText = tokenizedText.trim();
        return tokenizedText;
    }
}
